﻿using System;
using System.Collections.Generic;
using System.Text;
using Common.Utilities;

namespace Workflows.Components.TextMining.Clustering
{
    /// <summary>
    /// 
    /// </summary>
    public class FrequentTermSetCluster:IDocumentCluster
    {
        #region fields
        private IDistanceCalculator _DistanceCalculator;
        private IPurityCalculator _PurityCalculator;
        private Dictionary<int, double> _TermWeights;
        private int _NGramLen = 0;
        
        private Dictionary<int, Dictionary<int, DocTermFreq>> _DocTerms;
        private Dictionary<string, List<int>> _TermStemmings2;
        private Dictionary<int, string> _TermStemmings1;

        private Dictionary<string, List<int>> _FrequentTermSets;        // termid,termid,termid -> list of docids
        private Dictionary<string, double> _TermSetOverlaps;
        private Dictionary<int, ClusterNode> _ClusterResults;


        private bool _CanOverlap;
        private double _ReassociationDistanceRatio = 1.50;
        private double _MinClusterDistance = 0.10;
        private double _TermCoverageThreshold = 0.250;
        private double _TermFreqThreshold;
        #endregion

        #region IDocumentCluster Members

        public event ClusterStatusChanged OnClusterStatusChanged;

        public IDistanceCalculator DistanceCalculator
        {
            get { return _DistanceCalculator; }
            set { _DistanceCalculator=value; }
        }

        public IPurityCalculator PurityCalculator
        {
            get { return _PurityCalculator; }
            set { _PurityCalculator=value; }
        }

        public Dictionary<int, double> TermWeights
        {
            get { return _TermWeights; }
            set { _TermWeights=value; }
        }

        public bool CanOverlap
        {
            get { return _CanOverlap; }
            set { _CanOverlap = value; }
        }

        /// <summary>
        /// 
        /// </summary>
        public double ReassociationRange
        {
            get { return this._ReassociationDistanceRatio; }
            set { this._ReassociationDistanceRatio = value; }
        }

        public void SetThreshold(double termCoverageThreshold, double termFreqThreshold, double clusterDistanceThreshold)
        {
            this._TermCoverageThreshold = termCoverageThreshold;
            this._TermFreqThreshold = termFreqThreshold;
            this._MinClusterDistance = clusterDistanceThreshold;
        }

        public Dictionary<int, ClusterNode> ClusterDocuments(
            Dictionary<int, Dictionary<int, DocTermFreq>> docTerms,
            bool useNgram, int ngramLen)
        {
            if (useNgram)
                this._NGramLen = ngramLen;
            return this.ClusterDocuments(docTerms, null, null);
        }

        public Dictionary<int, ClusterNode> ClusterDocuments(
            Dictionary<int, Dictionary<int, DocTermFreq>> docTerms,
            Dictionary<int,string> termStemmings1,
            Dictionary<string,List<int>> termStemmings2)
        {
            if(this._PurityCalculator ==null)
                throw new Exception("Purity calculator must be initialized first");

            this._DocTerms = docTerms;
            this._TermStemmings1 = termStemmings1;
            this._TermStemmings2 = termStemmings2;

            this.GenerateFrequentTermSets();

            this._ClusterResults=new Dictionary<int, ClusterNode>();
            while (this._FrequentTermSets.Count>0)
            {
                string nextBestTermSetKey = this.GetNextPurestFrequentTermSet();
                ClusterNode clusterNode=new ClusterNode();
                clusterNode.ClusterID = this.NextClusterID();
                clusterNode.DocIDs = this._FrequentTermSets[nextBestTermSetKey];
                clusterNode.Distance = this.CalculateClusterDistance(clusterNode.DocIDs);
                Dictionary<int, double> supportiveTerms = new Dictionary<int, double>();
                int[] sharedTermIDs = IntUtil.ToArray(nextBestTermSetKey);
                foreach(int sharedTermID in sharedTermIDs)
                {
                    supportiveTerms.Add(sharedTermID,1.0);
                }
                clusterNode.SupportiveTermWeights = supportiveTerms;
                this._ClusterResults.Add(clusterNode.ClusterID,clusterNode);
                this._FrequentTermSets.Remove(nextBestTermSetKey);
            }

            this._ClusterResults = this.FilterClustersByMinimumDistance(this._ClusterResults, this._MinClusterDistance);

            return this._ClusterResults;
        }

        #endregion

        #region init
        /// <summary>
        /// Populate Frequent term set and its entropy
        /// term id is sorted in ascending order
        /// </summary>
        /// <returns></returns>
        private void GenerateFrequentTermSets()
        {
            this._TermSetOverlaps=new Dictionary<string, double>();
            Dictionary<int,List<int>> allTerms =new Dictionary<int, List<int>>();
            List<int> selectedTermIDs=new List<int>();
            foreach(int docID in this._DocTerms.Keys)
            {
                Dictionary<int, DocTermFreq> termFreqs = this._DocTerms[docID];
                foreach(int termID in termFreqs.Keys)
                {
                    if(allTerms.ContainsKey(termID))
                    {
                        List<int> docIDs = allTerms [termID];
                        docIDs.Add(docID);
                        allTerms [termID] = docIDs;
                    }
                    else
                    {
                        List<int> docIDs=new List<int>();
                        docIDs.Add(docID);
                        allTerms.Add(termID,docIDs);
                    }
                }
            }
            
            int[] allTermIDs=new int[allTerms.Count];
            allTerms.Keys.CopyTo(allTermIDs,0);
            List<int> sortedTermIDs=new List<int>();
            sortedTermIDs.AddRange(allTermIDs);
            sortedTermIDs.Sort();

            Dictionary<string,List<int>> termSetAdded=new Dictionary<string, List<int>>();
            foreach(int termID in sortedTermIDs)
            {
                double coverage = (double) allTerms[termID].Count/this._DocTerms.Count;
                if(coverage >= _TermCoverageThreshold)
                {
                    selectedTermIDs.Add(termID);
                    termSetAdded.Add(termID.ToString(),allTerms[termID]);
                }
            }

            this._FrequentTermSets = new Dictionary<string, List<int>>();
            while (termSetAdded.Count >0)
            {
                foreach(string key in termSetAdded.Keys)
                {
                    this._FrequentTermSets.Add(key, termSetAdded[key]);
                }
                string[] keys=new string[termSetAdded.Count];
                termSetAdded.Keys.CopyTo(keys,0);
                termSetAdded =new Dictionary<string, List<int>>();
                foreach (string concatedTermIDs in keys)
                {
                    List<int> termIDs = new List<int>();
                    termIDs.AddRange(IntUtil.ToArray(concatedTermIDs));
                    foreach (int termID in selectedTermIDs)
                    {
                        if (!termIDs.Contains(termID))
                        {
                            List<int> existingSupportingDocIDs = this._FrequentTermSets[concatedTermIDs];
                            List<int> currentSupportingDocIDs = this._FrequentTermSets[termID.ToString()];
                            List<int> sharedSupportingDocIDs = IntUtil.Intersection(existingSupportingDocIDs,
                                                                                    currentSupportingDocIDs);
                            if (sharedSupportingDocIDs != null && sharedSupportingDocIDs.Count > 0)
                            {
                                double support = (double) sharedSupportingDocIDs.Count/this._DocTerms.Count;
                                if (support >= _TermCoverageThreshold)
                                {
                                    List<int> newTermIDs = new List<int>();
                                    newTermIDs.AddRange(termIDs.ToArray());
                                    newTermIDs.Add(termID);
                                    newTermIDs.Sort();
                                    string newConcatenatedTermIDKey = IntUtil.ToString(newTermIDs.ToArray());
                                    if(!termSetAdded.ContainsKey(newConcatenatedTermIDKey))
                                    {
                                        termSetAdded.Add(newConcatenatedTermIDKey, sharedSupportingDocIDs);    
                                    }
                                }
                            }
                        }
                    }
                }
            }

            this._TermSetOverlaps=new Dictionary<string, double>();
            string[] termSetKeys = new string[this._FrequentTermSets.Count];
            this._FrequentTermSets.Keys.CopyTo(termSetKeys, 0);
            for (int i = 0; i < termSetKeys.Length; i++)
            {
                List<int> termSetIDs1 = new List<int>();
                termSetIDs1.AddRange(IntUtil.ToArray(termSetKeys[i]));
                double entropyFreq = 0;
                for (int k = 0; k < termSetKeys.Length; k++)
                {
                    if (k != i)
                    {
                        List<int> termSetIDs2 = new List<int>();
                        termSetIDs2.AddRange(IntUtil.ToArray(termSetKeys[k]));
                        if(IntUtil.Intercept(termSetIDs1,termSetIDs2))
                        {
                            int docCount2 = this._FrequentTermSets[termSetKeys[k]].Count;
                            double freq = 1.0/docCount2;
                            entropyFreq -= freq*Math.Log(freq, Math.E);
                        }
                    }
                }
                entropyFreq = entropyFreq/termSetIDs1.Count;
                this._TermSetOverlaps.Add(termSetKeys[i],entropyFreq);
            }
        }

        private int NextClusterID()
        {
            int lastClusterID = 0;
            if (this._ClusterResults.Count > 0)
            {
                foreach (int clusterID in this._ClusterResults.Keys)
                {
                    if (clusterID > lastClusterID)
                        lastClusterID = clusterID;
                }
            }
            return lastClusterID + 1;
        }

        private Dictionary<int, int> InitTermFrequency(Dictionary<int, DocTermFreq> termFreqs)
        {
            Dictionary<int, int> termFreq = new Dictionary<int, int>();
            if (termFreqs != null && termFreqs.Count > 0)
            {
                foreach (int termID in termFreqs.Keys)
                {
                    termFreq.Add(termID, termFreqs[termID].Count);
                }
            }
            return termFreq;
        }
        #endregion

        #region entropy
        private string GetNextPurestFrequentTermSet()
        {
            string[] keys=new string[this._FrequentTermSets.Count];
            this._FrequentTermSets.Keys.CopyTo(keys,0);
            string bestCandidateKey = keys[0];
            double bestEntropy = this._TermSetOverlaps[bestCandidateKey];
            if(keys.Length >1)
            {
                for(int i=1;i<keys.Length;i++)
                {
                    if (this._TermSetOverlaps[keys[i]] < bestEntropy)
                    {
                        bestCandidateKey = keys[i];
                        bestEntropy = this._TermSetOverlaps[keys[i]];
                    }
                }
            }

            return bestCandidateKey;
        }

        #endregion

        #region distance
        private double CalculateClusterDistance(List<int> clusteredDocIDs)
        {
            List<double> docDistances=new List<double>();
            for(int i=0;i<clusteredDocIDs.Count;i++)
            {
                Dictionary<int, DocTermFreq> termFreqs1 = this._DocTerms[clusteredDocIDs[i]];
                //Dictionary<int, int> termFreq1 = this.InitTermFrequency(termFreqs1);
                for(int k=i+1;k<clusteredDocIDs.Count;k++)
                {
                    Dictionary<int, DocTermFreq> termFreqs2 = this._DocTerms[clusteredDocIDs[k]];
                    //Dictionary<int, int> termFreq2 = this.InitTermFrequency(termFreqs2);
                    Dictionary<int, int> sharedTermIDs = new Dictionary<int, int>();
                    double distance;
                    if (this._TermWeights == null || this._TermWeights.Count == 0)
                        distance =
                            this._DistanceCalculator.CalculateDistance(
                                termFreqs1, termFreqs2, ref sharedTermIDs);
                    else if (this._TermStemmings1 == null || this._TermStemmings2 == null)
                        distance = this._DistanceCalculator.CalculateDistance(
                            termFreqs1, termFreqs2, this._TermWeights,
                            true, this._NGramLen, ref sharedTermIDs);
                    else
                        distance = this._DistanceCalculator.CalculateDistance(
                            termFreqs1, termFreqs2, TermWeights,
                            this._TermStemmings1, this._TermStemmings2, ref sharedTermIDs);
                    docDistances.Add(distance);
                }
            }
            return StatsUtil.GetAverage(docDistances).Value;
        }
        #endregion

        #region filter
        private Dictionary<int, ClusterNode> FilterClustersByMinimumDistance(Dictionary<int, ClusterNode> clusterResults, double minDistance)
        {
            List<int> selectedClusterIDs=new List<int>();
            foreach(int clusterID in clusterResults.Keys)
            {
                ClusterNode clusterNode = clusterResults[clusterID];
                if(clusterNode.Distance<=minDistance)
                {
                    selectedClusterIDs.Add(clusterID);
                }
            }

            Dictionary<int,ClusterNode> filteredClusterNodes=new Dictionary<int, ClusterNode>();
            foreach(int clusterID in selectedClusterIDs)
            {
                filteredClusterNodes.Add(clusterID, clusterResults[clusterID]);
            }
            return filteredClusterNodes;
        }
        #endregion
    }
}
